#include "Tokenizer.h"
#include <iostream>

using namespace std;

std::string Tokenizer::m_sDelimiters("\n\t\r ~!#%^&*()-=+{}\";:,<>,/\\?.'[]");

Tokenizer::Tokenizer(void)
{
    
}

Tokenizer::~Tokenizer(void)
{
}

void Tokenizer::tokenize(std::string inStream, bool inRetainDelimiters)// = true)
{
    string::size_type nDelimiterPos = 0;
    string::size_type nTokenPos = 0;
	bool bTrue = true; //Compiler warning
    Token xToken;

	while(bTrue)
	{
        nTokenPos = inStream.find_first_not_of(m_sDelimiters, nDelimiterPos);
        if(string::npos == nTokenPos)
            //At end of string/Not found...
        {
            if(true == inRetainDelimiters)
            //At end of string, copy rest as delimiters
            {
                for(unsigned int i=0;i<(inStream.length() - nDelimiterPos);i++)
                {
                    xToken.m_sValue = inStream.substr(nDelimiterPos+ i,1);
                    xToken.m_bIsDelimiter = true;
                    m_asTokens.push_back(xToken);        
                }
                break;
            }
        }

        if(true == inRetainDelimiters)
        {
            for(unsigned int i= 0;i<(nTokenPos - nDelimiterPos);i++)
            {
                xToken.m_sValue = inStream.substr(nDelimiterPos+ i,1);
                xToken.m_bIsDelimiter = true;
                m_asTokens.push_back(xToken);        
            }
        }

    	nDelimiterPos = inStream.find_first_of(m_sDelimiters, nTokenPos);
        if (string::npos == nDelimiterPos )
            //At end of string, copy rest as token
        {
            xToken.m_sValue = inStream.substr(nTokenPos, inStream.length() - nTokenPos);
            xToken.m_bIsDelimiter = false;
            m_asTokens.push_back(xToken);
            //This step does not seem to be needed; still, retaining
            break;
        }

        xToken.m_sValue = inStream.substr(nTokenPos,  nDelimiterPos - nTokenPos);
        xToken.m_bIsDelimiter = false;
    	m_asTokens.push_back(xToken);

		bTrue=true; //suppress warning
	
	}
    //TODO Small bug: sometimes extra line inserted at end of the file.
}


void Tokenizer::dump()
{
    int nListSize;
    nListSize = (int)m_asTokens.size();

    for(int nLooper = 0; nLooper<nListSize; nLooper++)
    {
        cout<<m_asTokens[nLooper].m_sValue;
        //cout<<"===";
        //cout<<endl;
    }


}